{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d8351622",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://huggingface.co/datasets/mesolitica/malaysian-twitter-by-topics/resolve/main/malaysia-topics.zip\n",
    "# !unzip malaysia-topics.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "94047b9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !mkdir twitter-topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "ff17982c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "250"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from glob import glob\n",
    "import json\n",
    "import random\n",
    "\n",
    "files = glob('twitter-topics/*.json')\n",
    "len(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a7ffd1de",
   "metadata": {},
   "outputs": [],
   "source": [
    "X1, X2, Y = [], [], []\n",
    "for f in files:\n",
    "    try:\n",
    "        with open(f) as fopen:\n",
    "            data = json.load(fopen)\n",
    "\n",
    "        data = random.sample(data, min(len(data), 20))\n",
    "        topic = os.path.split(f)[1].replace('.json', '')\n",
    "        X1.extend(data)\n",
    "        X2.extend([topic] * len(data))\n",
    "        Y.extend([1] * len(data))\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "9ffaa3a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4935, 4935, 4935)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X1), len(X2), len(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "080b2394",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('twitter-topics.json', 'w') as fopen:\n",
    "    json.dump({\n",
    "        'X1': X1,\n",
    "        'X2': X2,\n",
    "        'Y': Y\n",
    "    }, fopen)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
